/**
 * @author 李亚松
 * @email  lysong01@gmail.com
 * 
 * Segmenter.java 2012-11-30
 * description 
 */
package liyasong.chinese.analysis;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;

import java.io.FileReader;
import java.util.HashSet;
import java.util.Set;

import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

public class Segmenter {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		if (args.length != 1) {
			System.out.println("the args are wrong");
		} else {
			long start = System.currentTimeMillis();
			Reader reader;
			try {
				String fileName = "柴静";
				reader = new FileReader(new File("resources\\"+fileName+"Style.txt"));
//				reader = new StringReader("北理工北门进直行学生综合服务楼二层");
		
				IKSegmenter ik = new IKSegmenter(reader, true);
//				ik.reset(new StringReader("分词时只提取词库包含的词"));
				long start1 = System.currentTimeMillis();
				System.out.println("construct the IK costs:"+(start1-start)+"ms");
				Lexeme lexeme = null;
				FileWriter fWriter = new FileWriter(
						new File("resources\\"+fileName+"Words.txt"));
				int count = 0;
				Set<String> wordsSet = new HashSet<>();
				while ((lexeme = ik.next()) != null) {
//					System.out.print(lexeme.getLexemeText()+"|");
					String word = lexeme.getLexemeText();
					if (word.length() > 1 && word.getBytes().length!=word.length()) {
						fWriter.write(lexeme.getLexemeText()+"\n");
						count++;
						wordsSet.add(word);
					}
				}
				ik.reset(new FileReader(new File("resources\\纯银Style.txt")));
				while ((lexeme = ik.next()) != null) {
//					System.out.print(lexeme.getLexemeText()+"|");
					String word = lexeme.getLexemeText();
					if (word.length() > 1 && word.getBytes().length!=word.length()) {
						fWriter.write(lexeme.getLexemeText()+"\n");
						count++;
						wordsSet.add(word);
					}
				}
				System.out.println("the number of words:"+count);
				System.out.println("the size of set:"+wordsSet.size());
				fWriter.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} 
			
			System.out.println((System.currentTimeMillis()-start)+"ms");
		}	
	}

}
